{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "69486b28-5148-48ab-ae47-8ea58a7fb404",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install timm==1.0.9\n",
    "# !pip install albumentations==1.4.14\n",
    "# !pip install torcheval==0.0.7\n",
    "# !pip install pandas==2.2.2\n",
    "# !pip install numpy==1.26.4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "b0671889-2943-4530-856b-fa0415d66736",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/albumentations/__init__.py:13: UserWarning: A new version of Albumentations is available: 1.4.15 (you have 1.4.14). Upgrade using: pip install -U albumentations. To disable automatic update checks, set the environment variable NO_ALBUMENTATIONS_UPDATE to 1.\n",
      "  check_for_updates()\n"
     ]
    }
   ],
   "source": [
    "import sys, os, time, copy, gc\n",
    "import torch\n",
    "from torch import nn\n",
    "from torch.utils.data import DataLoader\n",
    "from pathlib import Path\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import albumentations as A\n",
    "from albumentations.pytorch import ToTensorV2\n",
    "import multiprocessing as mp\n",
    "\n",
    "from torcheval.metrics.functional import binary_auroc, multiclass_auroc\n",
    "\n",
    "from sklearn.model_selection import StratifiedGroupKFold\n",
    "\n",
    "import hashlib\n",
    "from joblib import Parallel, delayed\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from PIL import Image\n",
    "import torch.optim as optim\n",
    "\n",
    "from collections import defaultdict\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "sys.path.append('../src')\n",
    "from utils import set_seed, visualize_augmentations_positive, print_trainable_parameters\n",
    "from models import setup_model\n",
    "from training import fetch_scheduler, train_one_epoch, valid_one_epoch, run_training, get_nth_test_step\n",
    "from models import ISICModel, ISICModelEdgnet, setup_model\n",
    "from datasets import ISICDatasetSamplerW, ISICDatasetSampler, ISICDatasetSimple, ISICDatasetSamplerMulticlass, prepare_loaders\n",
    "from augmentations import get_augmentations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b7ff70a2-7e1d-48b5-9564-f18ca382d97a",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using device: cuda\n",
      "GPU: NVIDIA RTX A6000\n",
      "Number of GPUs: 1\n"
     ]
    }
   ],
   "source": [
    "# Set up device and random seed\n",
    "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\")\n",
    "print(f\"Using device: {device}\")\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    print(f\"GPU: {torch.cuda.get_device_name(0)}\")\n",
    "    print(f\"Number of GPUs: {torch.cuda.device_count()}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7803a16c-0893-4dee-98ac-ca017bde86fa",
   "metadata": {},
   "outputs": [],
   "source": [
    "original_data_path = \"../data/original\"\n",
    "original_root = Path('../data/original')\n",
    "\n",
    "data_artifacts = \"../data/artifacts\"\n",
    "os.makedirs(data_artifacts, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d762d6ea-d3bd-411a-8712-eaa1a01b178e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_2851924/2927513113.py:5: DtypeWarning: Columns (51,52) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  df_train = pd.read_csv(train_path)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of positive cases: 393\n",
      "Number of negative cases: 400666\n",
      "Ratio of negative to positive cases: 1019.51:1\n"
     ]
    }
   ],
   "source": [
    "# Set the HDF5 file path\n",
    "TRAIN_HDF5_FILE_PATH = original_root / 'train-image.hdf5'\n",
    "\n",
    "train_path = original_root / 'train-metadata.csv'\n",
    "df_train = pd.read_csv(train_path)\n",
    "df_train[\"path\"] = '../data/original/train-image/image/' + df_train['isic_id'] + \".jpg\"\n",
    "original_positive_cases = df_train['target'].sum()\n",
    "original_total_cases = len(df_train)\n",
    "original_positive_ratio = original_positive_cases / original_total_cases\n",
    "\n",
    "print(f\"Number of positive cases: {original_positive_cases}\")\n",
    "print(f\"Number of negative cases: {original_total_cases - original_positive_cases}\")\n",
    "print(f\"Ratio of negative to positive cases: {(original_total_cases - original_positive_cases) / original_positive_cases:.2f}:1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "dd11c7c5-040f-4170-984a-8123025bd9c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "MODEL_NAME = \"EVA\" # \"EDGENEXT\"\n",
    "\n",
    "\n",
    "CONFIG = {\n",
    "    \"seed\": 42 if MODEL_NAME == 'EVA' else 1997,\n",
    "    \"epochs\": 500,\n",
    "    \"img_size\": 336 if MODEL_NAME == 'EVA' else 256,\n",
    "    \"train_batch_size\": 32,\n",
    "    \"valid_batch_size\": 64,\n",
    "    \"learning_rate\": 1e-4,\n",
    "    \"scheduler\": 'CosineAnnealingLR',\n",
    "    \"min_lr\": 1e-6,\n",
    "    \"T_max\": 2000,\n",
    "    \"weight_decay\": 1e-6,\n",
    "    \"fold\" : 0,\n",
    "    \"n_fold\": 5,\n",
    "    \"n_accumulate\": 1,\n",
    "    \"group_col\": 'patient_id',\n",
    "    \"device\": device\n",
    "}\n",
    "\n",
    "model_name = \"eva02_small_patch14_336.mim_in22k_ft_in1k\" if MODEL_NAME == 'EVA' else \"edgenext_base.in21k_ft_in1k\"\n",
    "checkpoint_path = None\n",
    "\n",
    "\n",
    "if MODEL_NAME == 'EVA':\n",
    "    ISICModelPrep = ISICModel\n",
    "else:\n",
    "    ISICModelPrep = ISICModelEdgnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "03b6f4bd-2bbb-45ce-a8cb-d31dac99b553",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/pydantic/main.py:193: UserWarning: blur_limit and sigma_limit minimum value can not be both equal to 0. blur_limit minimum value changed to 3.\n",
      "  self.__pydantic_validator__.validate_python(data, self_instance=self)\n"
     ]
    }
   ],
   "source": [
    "data_transforms = get_augmentations(CONFIG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e655c2e6-541b-43d4-a9cb-ccec94436522",
   "metadata": {},
   "outputs": [],
   "source": [
    "def criterion(outputs, targets):\n",
    "    return nn.BCELoss()(outputs, targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "485bbd9d-135d-4e0d-9032-f4b805d344bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "# synthetic_custom_data = f\"../data/artifacts/syntetic_custom_base_{CONFIG['seed']}\"\n",
    "# os.makedirs(synthetic_custom_data, exist_ok=True)\n",
    "\n",
    "# tsp = StratifiedGroupKFold(2, shuffle=True, random_state=CONFIG['seed'])\n",
    "# metrics_ev_df = []\n",
    "# test_forecast = []\n",
    "# val_forecast = []\n",
    "# for fold_n, (train_index, val_index) in enumerate(tsp.split(df_train, y=df_train.target, groups=df_train[CONFIG[\"group_col\"]])):\n",
    "#     fold_df_train = df_train.iloc[train_index].reset_index(drop=True)\n",
    "#     fold_df_valid = df_train.iloc[val_index].reset_index(drop=True)\n",
    "#     synthetic_custom_data_pr = os.path.join(synthetic_custom_data, str(fold_n))\n",
    "#     os.makedirs(synthetic_custom_data_pr, exist_ok=True)\n",
    "\n",
    "#     for fn in fold_df_train[fold_df_train.target==1].isic_id.values:\n",
    "#         if fn not in images_to_include:\n",
    "#             continue\n",
    "#         img = Image.open(os.path.join('../data/original/train-image/image', fn + \".jpg\"))\n",
    "#         img.save(os.path.join(synthetic_custom_data_pr, fn + \".png\"))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "3241c3a9-8c31-454d-9813-c70b624addf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_name = f\"../models/oof_{{MODEL_NAME.lower()}}_base\"\n",
    "os.makedirs(folder_name, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "2b52d09c-81ca-4151-9988-85257da235d8",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metrics(drop_path_rate, drop_rate, models_folder, model_maker):\n",
    "    tsp = StratifiedGroupKFold(5, shuffle=True, random_state=CONFIG['seed'])\n",
    "    results_list = []\n",
    "    fold_df_valid_list = []\n",
    "    for fold_n, (train_index, val_index) in enumerate(tsp.split(df_train, y=df_train.target, groups=df_train[CONFIG[\"group_col\"]])):\n",
    "        fold_df_train = df_train.iloc[train_index].reset_index(drop=True)\n",
    "        fold_df_valid = df_train.iloc[val_index].reset_index(drop=True)\n",
    "        set_seed(CONFIG['seed'])\n",
    "        model = setup_model(model_name, drop_path_rate=drop_path_rate, drop_rate=drop_rate, model_maker=model_maker)\n",
    "        print_trainable_parameters(model)\n",
    "\n",
    "        train_loader, valid_loader = prepare_loaders(fold_df_train, fold_df_valid, CONFIG, data_transforms)\n",
    "    \n",
    "        optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n",
    "                           weight_decay=CONFIG['weight_decay'])\n",
    "        scheduler = fetch_scheduler(optimizer, CONFIG)\n",
    "    \n",
    "        model, history = run_training(\n",
    "            train_loader, valid_loader,\n",
    "            model, optimizer, scheduler,\n",
    "            device=CONFIG['device'],\n",
    "            num_epochs=CONFIG['epochs'],\n",
    "            CONFIG=CONFIG, \n",
    "            tolerance_max=20,\n",
    "            test_every_nth_step=lambda x: 5,\n",
    "            seed=CONFIG['seed'])\n",
    "        torch.save(model.state_dict(), os.path.join(models_folder, f\"model__{fold_n}\"))\n",
    "        results_list.append(np.max(history['Valid Kaggle metric']))\n",
    "\n",
    "        val_epoch_loss, val_epoch_auroc, val_epoch_custom_metric, tmp_predictions_all, tmp_targets_all = valid_one_epoch(\n",
    "            model, \n",
    "            valid_loader, \n",
    "            device=CONFIG['device'], \n",
    "            epoch=1, \n",
    "            optimizer=optimizer, \n",
    "            criterion=criterion, \n",
    "            use_custom_score=True,\n",
    "            metric_function=binary_auroc, \n",
    "            num_classes=1,\n",
    "            return_preds=True)\n",
    "\n",
    "        fold_df_valid['tmp_targets_all'] = tmp_targets_all\n",
    "        fold_df_valid['tmp_predictions_all'] = tmp_predictions_all\n",
    "        fold_df_valid['fold_n'] = fold_n\n",
    "        fold_df_valid_list.append(fold_df_valid)\n",
    "    fold_df_valid_list = pd.concat(fold_df_valid_list).reset_index(drop=True)\n",
    "    return results_list, fold_df_valid_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "091a2a0e-6b7a-4f99-87bd-41e35ee528ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainable params: 21744385 || all params: 21744385 || trainable%: 100.00\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:08<00:00,  2.14it/s, Epoch=1, LR=0.0001, Train_Auroc=0.582, Train_Loss=0.754]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:06<00:00,  2.72it/s, Epoch=2, LR=9.99e-5, Train_Auroc=0.751, Train_Loss=0.676]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:07<00:00,  2.71it/s, Epoch=3, LR=9.98e-5, Train_Auroc=0.825, Train_Loss=0.546]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:07<00:00,  2.68it/s, Epoch=4, LR=9.96e-5, Train_Auroc=0.853, Train_Loss=0.523]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:07<00:00,  2.71it/s, Epoch=5, LR=9.94e-5, Train_Auroc=0.86, Train_Loss=0.551] \n",
      "100%|██████████| 1112/1112 [04:03<00:00,  4.57it/s, Epoch=5, LR=9.94e-5, Valid_Auroc=0.526, Valid_Loss=0.583]\n",
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Validation AUROC Improved (-inf ---> 0.10594036364080138)\n",
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:07<00:00,  2.57it/s, Epoch=6, LR=9.92e-5, Train_Auroc=0.877, Train_Loss=0.466]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:07<00:00,  2.66it/s, Epoch=7, LR=9.89e-5, Train_Auroc=0.862, Train_Loss=0.491]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:07<00:00,  2.66it/s, Epoch=8, LR=9.86e-5, Train_Auroc=0.88, Train_Loss=0.461] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:07<00:00,  2.68it/s, Epoch=9, LR=9.82e-5, Train_Auroc=0.884, Train_Loss=0.444]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 19/19 [00:07<00:00,  2.62it/s, Epoch=10, LR=9.78e-5, Train_Auroc=0.892, Train_Loss=0.448]\n",
      "100%|██████████| 1214/1214 [04:26<00:00,  4.56it/s, Epoch=45, LR=5.82e-5, Valid_Auroc=0.521, Valid_Loss=0.581]\n",
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.57it/s, Epoch=46, LR=5.67e-5, Train_Auroc=0.924, Train_Loss=0.408]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.53it/s, Epoch=47, LR=5.52e-5, Train_Auroc=0.931, Train_Loss=0.357]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.65it/s, Epoch=48, LR=5.36e-5, Train_Auroc=0.935, Train_Loss=0.34] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.56it/s, Epoch=49, LR=5.21e-5, Train_Auroc=0.936, Train_Loss=0.343]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.60it/s, Epoch=50, LR=5.05e-5, Train_Auroc=0.94, Train_Loss=0.324] \n",
      "100%|██████████| 1214/1214 [04:26<00:00,  4.56it/s, Epoch=50, LR=5.05e-5, Valid_Auroc=0.521, Valid_Loss=0.207]\n",
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.63it/s, Epoch=51, LR=4.89e-5, Train_Auroc=0.918, Train_Loss=0.381]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.59it/s, Epoch=52, LR=4.74e-5, Train_Auroc=0.942, Train_Loss=0.337]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.55it/s, Epoch=53, LR=4.58e-5, Train_Auroc=0.931, Train_Loss=0.348]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.55it/s, Epoch=54, LR=4.43e-5, Train_Auroc=0.947, Train_Loss=0.304]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.64it/s, Epoch=55, LR=4.28e-5, Train_Auroc=0.938, Train_Loss=0.344]\n",
      "100%|██████████| 1214/1214 [04:26<00:00,  4.56it/s, Epoch=55, LR=4.28e-5, Valid_Auroc=0.52, Valid_Loss=0.204] \n",
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.61it/s, Epoch=56, LR=4.12e-5, Train_Auroc=0.942, Train_Loss=0.316]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.60it/s, Epoch=57, LR=3.97e-5, Train_Auroc=0.931, Train_Loss=0.347]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.62it/s, Epoch=58, LR=3.82e-5, Train_Auroc=0.933, Train_Loss=0.333]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.65it/s, Epoch=59, LR=3.67e-5, Train_Auroc=0.939, Train_Loss=0.319]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.56it/s, Epoch=60, LR=3.52e-5, Train_Auroc=0.952, Train_Loss=0.295]\n",
      "100%|██████████| 1214/1214 [04:27<00:00,  4.54it/s, Epoch=60, LR=3.52e-5, Valid_Auroc=0.521, Valid_Loss=0.308]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n",
      "100%|██████████| 20/20 [00:07<00:00,  2.52it/s, Epoch=61, LR=3.37e-5, Train_Auroc=0.945, Train_Loss=0.302]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.58it/s, Epoch=62, LR=3.23e-5, Train_Auroc=0.947, Train_Loss=0.3]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.55it/s, Epoch=63, LR=3.08e-5, Train_Auroc=0.942, Train_Loss=0.315]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.61it/s, Epoch=64, LR=2.94e-5, Train_Auroc=0.945, Train_Loss=0.315]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.55it/s, Epoch=65, LR=2.8e-5, Train_Auroc=0.958, Train_Loss=0.268] \n",
      "100%|██████████| 1214/1214 [04:26<00:00,  4.55it/s, Epoch=65, LR=2.8e-5, Valid_Auroc=0.521, Valid_Loss=0.343]\n",
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.58it/s, Epoch=66, LR=2.67e-5, Train_Auroc=0.953, Train_Loss=0.295]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.65it/s, Epoch=67, LR=2.53e-5, Train_Auroc=0.948, Train_Loss=0.294]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.55it/s, Epoch=68, LR=2.4e-5, Train_Auroc=0.948, Train_Loss=0.287] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.59it/s, Epoch=69, LR=2.27e-5, Train_Auroc=0.96, Train_Loss=0.267] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.58it/s, Epoch=70, LR=2.14e-5, Train_Auroc=0.956, Train_Loss=0.264]\n",
      "100%|██████████| 1214/1214 [04:26<00:00,  4.56it/s, Epoch=70, LR=2.14e-5, Valid_Auroc=0.521, Valid_Loss=0.403]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n",
      "100%|██████████| 20/20 [00:07<00:00,  2.57it/s, Epoch=71, LR=2.02e-5, Train_Auroc=0.954, Train_Loss=0.278]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.63it/s, Epoch=72, LR=1.89e-5, Train_Auroc=0.957, Train_Loss=0.273]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.59it/s, Epoch=73, LR=1.78e-5, Train_Auroc=0.971, Train_Loss=0.245]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.56it/s, Epoch=74, LR=1.66e-5, Train_Auroc=0.959, Train_Loss=0.27] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.55it/s, Epoch=75, LR=1.55e-5, Train_Auroc=0.967, Train_Loss=0.241]\n",
      "100%|██████████| 1214/1214 [04:26<00:00,  4.56it/s, Epoch=75, LR=1.55e-5, Valid_Auroc=0.521, Valid_Loss=0.334]\n",
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.56it/s, Epoch=76, LR=1.44e-5, Train_Auroc=0.962, Train_Loss=0.246]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.58it/s, Epoch=77, LR=1.34e-5, Train_Auroc=0.966, Train_Loss=0.23] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.65it/s, Epoch=78, LR=1.24e-5, Train_Auroc=0.972, Train_Loss=0.217]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.64it/s, Epoch=79, LR=1.14e-5, Train_Auroc=0.968, Train_Loss=0.238]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.54it/s, Epoch=80, LR=1.05e-5, Train_Auroc=0.965, Train_Loss=0.241]\n",
      "100%|██████████| 1214/1214 [04:26<00:00,  4.56it/s, Epoch=80, LR=1.05e-5, Valid_Auroc=0.521, Valid_Loss=0.379]\n",
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.64it/s, Epoch=81, LR=9.56e-6, Train_Auroc=0.97, Train_Loss=0.221] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.53it/s, Epoch=82, LR=8.71e-6, Train_Auroc=0.961, Train_Loss=0.256]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.54it/s, Epoch=83, LR=7.89e-6, Train_Auroc=0.96, Train_Loss=0.254] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.64it/s, Epoch=84, LR=7.12e-6, Train_Auroc=0.98, Train_Loss=0.191] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.58it/s, Epoch=85, LR=6.4e-6, Train_Auroc=0.971, Train_Loss=0.211] \n",
      "100%|██████████| 1214/1214 [04:48<00:00,  4.20it/s, Epoch=85, LR=6.4e-6, Valid_Auroc=0.521, Valid_Loss=0.266]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n",
      "100%|██████████| 20/20 [00:08<00:00,  2.44it/s, Epoch=86, LR=5.71e-6, Train_Auroc=0.971, Train_Loss=0.225]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.30it/s, Epoch=87, LR=5.07e-6, Train_Auroc=0.97, Train_Loss=0.207] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.44it/s, Epoch=88, LR=4.48e-6, Train_Auroc=0.971, Train_Loss=0.224]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.26it/s, Epoch=89, LR=3.93e-6, Train_Auroc=0.974, Train_Loss=0.198]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.46it/s, Epoch=90, LR=3.42e-6, Train_Auroc=0.972, Train_Loss=0.207]\n",
      "100%|██████████| 1214/1214 [05:02<00:00,  4.01it/s, Epoch=90, LR=3.42e-6, Valid_Auroc=0.521, Valid_Loss=0.36] \n",
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.52it/s, Epoch=91, LR=2.97e-6, Train_Auroc=0.971, Train_Loss=0.221]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.28it/s, Epoch=92, LR=2.56e-6, Train_Auroc=0.978, Train_Loss=0.2]  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.38it/s, Epoch=93, LR=2.19e-6, Train_Auroc=0.984, Train_Loss=0.163]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.29it/s, Epoch=94, LR=1.88e-6, Train_Auroc=0.96, Train_Loss=0.251] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.45it/s, Epoch=95, LR=1.61e-6, Train_Auroc=0.983, Train_Loss=0.174]\n",
      "100%|██████████| 1214/1214 [05:01<00:00,  4.03it/s, Epoch=95, LR=1.61e-6, Valid_Auroc=0.521, Valid_Loss=0.289]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n",
      "100%|██████████| 20/20 [00:08<00:00,  2.48it/s, Epoch=96, LR=1.39e-6, Train_Auroc=0.973, Train_Loss=0.209]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.34it/s, Epoch=97, LR=1.22e-6, Train_Auroc=0.97, Train_Loss=0.221] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.40it/s, Epoch=98, LR=1.1e-6, Train_Auroc=0.972, Train_Loss=0.211] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.26it/s, Epoch=99, LR=1.02e-6, Train_Auroc=0.973, Train_Loss=0.213]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.51it/s, Epoch=100, LR=1e-6, Train_Auroc=0.971, Train_Loss=0.213]   \n",
      "100%|██████████| 1214/1214 [05:03<00:00,  4.00it/s, Epoch=100, LR=1e-6, Valid_Auroc=0.521, Valid_Loss=0.304]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n",
      "100%|██████████| 20/20 [00:08<00:00,  2.42it/s, Epoch=101, LR=1.02e-6, Train_Auroc=0.967, Train_Loss=0.229]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.32it/s, Epoch=102, LR=1.1e-6, Train_Auroc=0.979, Train_Loss=0.186] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.39it/s, Epoch=103, LR=1.22e-6, Train_Auroc=0.965, Train_Loss=0.228]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.33it/s, Epoch=104, LR=1.39e-6, Train_Auroc=0.977, Train_Loss=0.202]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.53it/s, Epoch=105, LR=1.61e-6, Train_Auroc=0.98, Train_Loss=0.183] \n",
      "100%|██████████| 1214/1214 [05:02<00:00,  4.01it/s, Epoch=105, LR=1.61e-6, Valid_Auroc=0.521, Valid_Loss=0.293]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n",
      "100%|██████████| 20/20 [00:08<00:00,  2.49it/s, Epoch=106, LR=1.88e-6, Train_Auroc=0.977, Train_Loss=0.189]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.34it/s, Epoch=107, LR=2.19e-6, Train_Auroc=0.979, Train_Loss=0.191]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.42it/s, Epoch=108, LR=2.56e-6, Train_Auroc=0.974, Train_Loss=0.205]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.51it/s, Epoch=109, LR=2.97e-6, Train_Auroc=0.984, Train_Loss=0.178]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.32it/s, Epoch=110, LR=3.42e-6, Train_Auroc=0.973, Train_Loss=0.205]\n",
      "100%|██████████| 1214/1214 [05:03<00:00,  4.01it/s, Epoch=110, LR=3.42e-6, Valid_Auroc=0.521, Valid_Loss=0.387]\n",
      "/opt/conda/lib/python3.11/site-packages/torch/optim/lr_scheduler.py:1063: UserWarning: To get the last learning rate computed by the scheduler, please use `get_last_lr()`.\n",
      "  _warn_get_lr_called_within_step(self)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:08<00:00,  2.47it/s, Epoch=111, LR=3.93e-6, Train_Auroc=0.979, Train_Loss=0.194]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.56it/s, Epoch=112, LR=4.48e-6, Train_Auroc=0.983, Train_Loss=0.181]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 20/20 [00:07<00:00,  2.50it/s, Epoch=113, LR=5.07e-6, Train_Auroc=0.972, Train_Loss=0.21] \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/20 [00:00<?, ?it/s]"
     ]
    }
   ],
   "source": [
    "base_metrics, oof_forecasts = get_metrics(drop_path_rate=0, drop_rate=0, models_folder=folder_name, model_maker=ISICModelPrep)\n",
    "oof_forecasts.to_parquet(f'../data/artifacts/oof_forecasts_{MODEL_NAME.lower()}_base.parquet')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d98e33df-f941-4cca-adda-724e9b9742bc",
   "metadata": {},
   "outputs": [],
   "source": [
    "1"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9798265c-7924-453b-9dfa-b26458f64a0a",
   "metadata": {},
   "source": [
    "# Train with synthetic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6e9197d6-75ff-4e3d-af64-785a6bd7cb53",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_metrics_synth(drop_path_rate, drop_rate, models_folder, model_maker, synth_path = \"../data/artifacts/syntetic_base_folds_final\"):\n",
    "    tsp = StratifiedGroupKFold(5, shuffle=True, random_state=CONFIG['seed'])\n",
    "    results_list = []\n",
    "    fold_df_valid_list = []\n",
    "    for fold_n, (train_index, val_index) in enumerate(tsp.split(df_train, y=df_train.target, groups=df_train[CONFIG[\"group_col\"]])):\n",
    "        fold_df_train = df_train.iloc[train_index].reset_index(drop=True)\n",
    "        fold_df_valid = df_train.iloc[val_index].reset_index(drop=True)\n",
    "        set_seed(CONFIG['seed'])\n",
    "        model = setup_model(model_name, drop_path_rate=drop_path_rate, drop_rate=drop_rate, model_maker=model_maker)\n",
    "        print_trainable_parameters(model)\n",
    "\n",
    "        synthetic_Df = pd.concat([\n",
    "            pd.DataFrame({\n",
    "                \"path\": glob.glob(f\"{synth_path}/{CONFIG['seed']}/{fold_n}/hr/**.png\")\n",
    "            }),\n",
    "            pd.DataFrame({\n",
    "                \"path\": glob.glob(f\"{synth_path}/{CONFIG['seed']}/{fold_n}/lr/**.png\")\n",
    "            })\n",
    "        ])\n",
    "        \n",
    "        synthetic_Df['weight'] = 1\n",
    "        synthetic_Df['target'] = synthetic_Df.path.apply(lambda x: int(x.split('___')[1].split('.')[0]))\n",
    "\n",
    "        synthetic_Df = pd.concat([\n",
    "            synthetic_Df, fold_df_train[['path', 'target', 'weight']].reset_index(drop=True)\n",
    "        ]).reset_index(drop=True)\n",
    "\n",
    "        \n",
    "        train_loader, valid_loader = prepare_loaders(synthetic_Df, fold_df_valid, CONFIG, data_transforms)\n",
    "    \n",
    "        optimizer = optim.Adam(model.parameters(), lr=CONFIG['learning_rate'], \n",
    "                           weight_decay=CONFIG['weight_decay'])\n",
    "        scheduler = fetch_scheduler(optimizer, CONFIG)\n",
    "    \n",
    "        model, history = run_training(\n",
    "            train_loader, valid_loader,\n",
    "            model, optimizer, scheduler,\n",
    "            device=CONFIG['device'],\n",
    "            num_epochs=CONFIG['epochs'],\n",
    "            CONFIG=CONFIG, \n",
    "            tolerance_max=20,\n",
    "            test_every_nth_step=lambda x: 5,\n",
    "            seed=CONFIG['seed'])\n",
    "        torch.save(model.state_dict(), os.path.join(models_folder, f\"model__{fold_n}\"))\n",
    "        results_list.append(np.max(history['Valid Kaggle metric']))\n",
    "\n",
    "        val_epoch_loss, val_epoch_auroc, val_epoch_custom_metric, tmp_predictions_all, tmp_targets_all = valid_one_epoch(\n",
    "            model, \n",
    "            valid_loader, \n",
    "            device=CONFIG['device'], \n",
    "            epoch=1, \n",
    "            optimizer=optimizer, \n",
    "            criterion=criterion, \n",
    "            use_custom_score=True,\n",
    "            metric_function=binary_auroc, \n",
    "            num_classes=1,\n",
    "            return_preds=True)\n",
    "\n",
    "        fold_df_valid['tmp_targets_all'] = tmp_targets_all\n",
    "        fold_df_valid['tmp_predictions_all'] = tmp_predictions_all\n",
    "        fold_df_valid['fold_n'] = fold_n\n",
    "        fold_df_valid_list.append(fold_df_valid)\n",
    "    fold_df_valid_list = pd.concat(fold_df_valid_list).reset_index(drop=True)\n",
    "    return results_list, fold_df_valid_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fae0bc25-ac32-40fa-b5bd-61229abf438b",
   "metadata": {},
   "outputs": [],
   "source": [
    "folder_name = f\"../models/oof_{{MODEL_NAME.lower()}}_base__synth\"\n",
    "os.makedirs(folder_name, exist_ok=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "480be65a-e8ed-4ca3-afa7-ca5ac666fab4",
   "metadata": {},
   "outputs": [],
   "source": [
    "base_metrics, oof_forecasts = get_metrics(drop_path_rate=0, drop_rate=0, models_folder=folder_name, model_maker=ISICModelPrep)\n",
    "oof_forecasts.to_parquet(f'../data/artifacts/oof_forecasts_{MODEL_NAME.lower()}_base__synth.parquet')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
